package edu.hit.pact.pgse.crawler.bbs;

import java.net.MalformedURLException;
import java.net.URL;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import edu.hit.pact.pgse.bean.RawWebPage;
import edu.hit.pact.pgse.crawler.download.FetchorFactory;
import edu.hit.pact.pgse.crawler.download.WebPageFetcher;

import edu.hit.pact.pgse.crawler.store.WebPageStore;
import edu.hit.pact.pgse.crawler.util.Anchor;
import edu.hit.pact.pgse.crawler.util.DuplicateUrlFilter;

import edu.hit.pact.pgse.util.ChineseCharEncode;
import edu.hit.pact.pgse.util.Md5Sum;
import edu.hit.pact.pgse.util.Utilities;

import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;




public class BBSArticleFetchProcessor extends AbstractBBSFetchorThread {
	
	private String subitem;

	//private String md5Sum;
	private String articleName;
	private static final String nextPageStr = ChineseCharEncode.encode("下一页");
	private int sumSuccessedPageNum = 0;
	private DuplicateUrlFilter aritclePageDuplicateUrlFilter; //避免一个帖子内部页面的重复
	
	private Log articleLogger;
	private boolean isShedule;
	//private List updateList;
	
	private ConcurrentArrayList<ArticleUpdateInfo> updateList ;
	
	public BBSArticleFetchProcessor(Log logger, WebPageFetcher fetcher,WebPageStore webPageStore,
			ConcurrentLinkedQueue<URL> articleUrlQueue, String threadName,String subitem) {
		
		super(logger, fetcher, webPageStore, articleUrlQueue, threadName);
		this.subitem = subitem;
		this.aritclePageDuplicateUrlFilter = new DuplicateUrlFilter();
		this.articleLogger = LogFactory.getLog("articleList");
	}
	
	public BBSArticleFetchProcessor(Log logger, WebPageFetcher fetcher,WebPageStore webPageStore,
			ConcurrentLinkedQueue<Anchor> articleAnchorQueue, String threadName,String subitem,AtomicBoolean isFinished) {
		
		super(logger, fetcher, webPageStore, articleAnchorQueue, threadName,isFinished);
		this.subitem = subitem;
		this.aritclePageDuplicateUrlFilter = new DuplicateUrlFilter();
		this.articleLogger = LogFactory.getLog("articleList");
	}
	
	public BBSArticleFetchProcessor(Log logger, WebPageFetcher fetcher,
			WebPageStore webPageStore,
			ConcurrentLinkedQueue<Anchor> articleAnchorQueue, String threadName,
			String subitem, AtomicBoolean isFinished,
			ConcurrentArrayList<ArticleUpdateInfo> updateList,boolean isShedule) {

		this(logger,fetcher,webPageStore,articleAnchorQueue,threadName,subitem,isFinished);
		this.updateList = updateList;
		this.isShedule = isShedule;
	}

	
	
	public void setUpdateList(ConcurrentArrayList<ArticleUpdateInfo> updateList) {
		this.updateList = updateList;
	}

	@Override
	public void mainRun() throws MalformedURLException, InterruptedException {
		
		int tryTimes = 0;
		int tryGetHomePageTimes = 0;
		while (true) {
			
//			if((this.successedPageNum != 0 && this.successedPageNum % 5 == 0) || (this.sumSuccessedPageNum > 50)){
//				this.fetcher = buildWebPageFetcher();
//				this.sumSuccessedPageNum = 0;
//			}	
			if(shedule(isShedule)){
				this.fetcher = buildWebPageFetcher();
			}
			this.aritclePageDuplicateUrlFilter.clear();
//		    homepage = this.articleUrlQueue.poll();
//			logger.error(this.getName() + " get the homepage :" + homepage);
//			
//			tryTimes = 0;
//			tryGetHomePageTimes = 0;
//			if(homepage == null && this.isFinished.get()){
//				break;
//			}
//			while(homepage == null){
//				tryGetHomePageTimes ++;
////				if(tryGetHomePageTimes > 10){
////					break;
////				}
//				sleep(1000);
//				homepage = this.articleUrlQueue.poll();
//			}
//			
//			if(null == homepage){
//				logger.error(this.getName() + " cannot get the homepage from the articleQueue.");
//				break;
//			}
			
			Anchor anchorHomePage = this.articleAnchorQueue.poll();
			tryTimes = 0;
			tryGetHomePageTimes=0;
			
			if(null == anchorHomePage && this.isFinished.get()){
				break;
			}
			
			while(null == anchorHomePage){
				tryGetHomePageTimes++;
				sleep(1000);
				anchorHomePage = this.articleAnchorQueue.poll();
			}
			homepage = anchorHomePage.getUrl();
			articleName = Utilities.filterFileName(anchorHomePage.getText());
			
			//System.out.println("articleName = " + articleName);
			logger.info(this.getName() + "begin to fetch the "+this.successedPageNum+"th homepage:"+homepage.toString());
			RawWebPage articlePage = this.fetcher.fetchURL(homepage, 3, null, null, null);
			logger.info(this.getName() + "finished to fetch the "+this.successedPageNum+"th homepage:"+homepage.toString());
			while (articlePage == null) {
				tryTimes ++;
				if(tryTimes > 3){
					break;
				}
				logger.info(this.getName() + "failed to fetch the "
						+ this.successedPageNum + "th homepage :"
						+ homepage.toString() + "and will try the " + tryTimes
						+ " times");
				articlePage = this.fetcher.fetchURL(homepage, 3, null, null, null);
				logger.info(this.getName() + "finished to refetch("+tryTimes+"th) the homepage :" + homepage.toString());
			}
			if(null == articlePage){
				logger.info(this.getName() + " cannot fetch the " + this.successedPageNum+"th homepage.");
				continue;
			}
			this.aritclePageDuplicateUrlFilter.contains(homepage.toExternalForm());
			fetchOneArticle(homepage,articlePage);
			this.successedPageNum ++;
	}
}

	public void fetchOneArticle(URL homepage, RawWebPage articlePage) {
		ArticleUpdateInfo articleInfo = new ArticleUpdateInfo();
		articleInfo.setHomepage(homepage);
		//md5Sum = Md5Sum.changeTo18String(homepage.toExternalForm());
		//String md5name = md5Sum;
        String md5name = this.getName() + "-" + (this.successedPageNum+1) + "-" + articleName;
		int pageNum = 1;
		logger.info(this.getName() + " : dowload the "
				+ (this.getSuccessedPageNum() + 1) + "-" + pageNum
				+ "th page :" + homepage.toString());
		int tryStoreTimes = 0;
		while(tryStoreTimes < 2 && !this.webPageStore.storeWebPage(articlePage, subitem + "/" + md5name,
				pageNum+"", true)){
			//this.fetcher = FetchorFactory.buildWebPageFetcher();
			tryStoreTimes++;
		}
		pageNum++;
		tryStoreTimes = 0;
		
		//this.webPageStore.storeWebPage(articlePage, subitem + "/" + md5name,pageNum, true);
        logger.error(this.getName() + " has saved " + (this.successedPageNum+1) + " articles." );
		HashMap<String, URL> linksAnchor = new HashMap<String, URL>();
		linksAnchor = articlePage.getLinksAnchor();
		boolean isLastPage = false;
		URL nextPage = null;
		if (!linksAnchor.containsKey(nextPageStr)) {
			isLastPage = true;
		}else{
			nextPage = linksAnchor.get(nextPageStr);
			logger.info(this.getName() + " contain next page tag and the next page is nextPage is : " + nextPage.toString() );
		}
		
		while (!isLastPage) {
			if(this.sumSuccessedPageNum > 50){
				this.fetcher = buildWebPageFetcher();
				this.sumSuccessedPageNum = 0;
			}
			if(this.aritclePageDuplicateUrlFilter.contains(nextPage.toExternalForm())){
				logger.info(this.getName() + " maybe in dead loop,so give up to fetch the article:" + nextPage.toExternalForm());
				break;
			}
			RawWebPage nextArticlePage = null;
			int tryTimes = 0;
			while (nextArticlePage == null) {
				//this.fetcher = FetchorFactory.buildWebPageFetcher();
				logger.info(this.getName() + " begin to fetch " +(this.successedPageNum + 1)+ "-" + pageNum + "th page : "+ nextPage.toString());
				nextArticlePage = this.fetcher.fetchURL(nextPage, 3, null,null, null);
				logger.info(this.getName() + " finish to fecth "  +(this.successedPageNum + 1) + "-" + pageNum+ "th page : "+ nextPage.toString());
				tryTimes++;
				if (tryTimes > 3) {
					return;
				}
			}
			logger.info(this.getName() + " : dowload the "
					+ (this.getSuccessedPageNum() + 1) + "-" + pageNum
					+ "th page :" + nextPage.toString());

			tryStoreTimes = 0;
			while(tryStoreTimes < 2 && !this.webPageStore.storeWebPage(nextArticlePage, subitem + "/"
					+ md5name, pageNum+"", true)){
				tryStoreTimes ++;
			}
			//this.webPageStore.storeWebPage(nextArticlePage, subitem + "/"+ md5name, pageNum, true);
			pageNum++;
			this.sumSuccessedPageNum ++;
			linksAnchor = nextArticlePage.getLinksAnchor();
			if (!linksAnchor.containsKey(nextPageStr)) {
				isLastPage = true;
			}else{
				nextPage = linksAnchor.get(nextPageStr);
				logger.info(this.getName() + " contain next page tag and the next page is nextPage is : " + nextPage.toString() );
			}
		}
		
		if(nextPage == null){
			nextPage = homepage;
		}
		
		/*if need to update,remove the comment
		 * articleInfo.setPageNum(pageNum);
		articleInfo.setLastpage(nextPage);
		articleInfo.setFolderName(subitem + "/" + md5name);
		this.updateList.add(articleInfo);
		*/
		this.articleLogger.info(this.getName() +" " + this.homepage.toExternalForm() + " "  + nextPage.toExternalForm() + " " + pageNum );
		
		
		
		logger.info(this.getName() + " : the "
				+ (this.getSuccessedPageNum() + 1) + "th article has  " + pageNum
				+ " pages.");
		logger.info(this.getName() + " has completed the "
				+ (this.getSuccessedPageNum() + 1) + "th article.");
	}
	

}
